import torch
import numpy as np
import gymnasium as gym
from typing import Tuple

'''[ElegantRL.2022.12.12](github.com/AI4Fiance-Foundation/ElegantRL)'''

Array = np.ndarray
TEN = torch.Tensor

InstallGymnasiumBox2D = """Install gymnasium[Box2D]
# LinuxOS (Ubuntu) 
sudo apt update && sudo apt install swig
python3 -m pip install --upgrade pip --no-warn-script-location
pip3 install -i http://pypi.douban.com/simple/ --trusted-host pypi.douban.com --user gymnasium gymnasium[Box2D]

# WindowOS (Win10)
python -m pip install --upgrade pip
pip3 install swig gymnasium gymnasium[Box2D]
"""
ARY = np.ndarray


class PendulumEnv(gym.Wrapper):  # a demo of custom env
    def __init__(self):
        gym_env_name = 'Pendulum-v1'
        super().__init__(env=gym.make(gym_env_name))

        '''the necessary env information when you design a custom env'''
        self.env_name = gym_env_name  # the name of this env.
        self.state_dim = self.observation_space.shape[0]  # feature number of state
        self.action_dim = self.action_space.shape[0]  # feature number of action
        self.if_discrete = False  # discrete action or continuous action

    def reset(self, **kwargs) -> Tuple[ARY, dict]:  # reset the agent in env
        state, info_dict = self.env.reset()
        return state, info_dict

    def step(self, action: ARY) -> Tuple[ARY, float, bool, bool, dict]:  # agent interacts in env
        # OpenAI Pendulum env set its action space as (-2, +2). It is bad.
        # We suggest that adjust action space to (-1, +1) when designing a custom env.
        state, reward, terminated, truncated, info_dict = self.env.step(action * 2)
        state = state.reshape(self.state_dim)
        return state, float(reward) * 0.5, terminated, truncated, info_dict

# PLAN TODO
# class GymNormaEnv(gym.Wrapper):
#     def __init__(self, env_name: str = 'Hopper-v3'):
#         gym.logger.set_level(40)  # Block warning
#         super(GymNormaEnv, self).__init__(env=gym.make(env_name))
#
#         if env_name == 'Hopper-v3':
#             self.env_num = 1
#             self.env_name = env_name
#             self.max_step = 1000
#             self.state_dim = 11
#             self.action_dim = 3
#             self.if_discrete = False
#             self.target_return = 3000
#
#             # 4 runs
#             self.state_avg = torch.tensor([1.3819, -0.0105, -0.3804, -0.1759, 0.1959, 2.4185, -0.0406, -0.0172,
#                                            -0.1465, -0.0450, -0.1616], dtype=torch.float32)
#             self.state_std = torch.tensor([0.1612, 0.0747, 0.2357, 0.1889, 0.6431, 0.6253, 1.4806, 1.1569, 2.2850,
#                                            2.2124, 6.5147], dtype=torch.float32)
#         elif env_name == 'Swimmer-v3':
#             self.env_num = 1
#             self.env_name = env_name
#             self.max_step = 1000
#             self.state_dim = 8
#             self.action_dim = 2
#             self.if_discrete = False
#             self.target_return = 360.0
#
#             # self.state_avg = th.zeros(1, dtype=th.float32)
#             # self.state_std = th.ones(1, dtype=th.float32)
#             # 6 runs
#             self.state_avg = torch.tensor([0.5877, -0.2745, -0.2057, 0.0802, 0.0105, 0.0158, -0.0047, -0.0057],
#                                           dtype=torch.float32)
#             self.state_std = torch.tensor([0.5324, 0.5573, 0.5869, 0.4787, 0.5617, 0.8538, 1.2658, 1.4649],
#                                           dtype=torch.float32)
#         elif env_name == 'Ant-v3':
#             self.env_num = 1
#             self.env_name = env_name
#             self.max_step = 1000
#             self.state_dim = 17
#             self.action_dim = 6
#             self.if_discrete = False
#             self.target_return = 5000
#
#             # self.state_avg = th.zeros(1, dtype=th.float32)
#             # self.state_std = th.ones(1, dtype=th.float32)
#             # 2 runs
#             self.state_avg = torch.tensor([6.3101e-01, 9.3039e-01, 1.1357e-02, -6.0412e-02, -1.9220e-01,
#                                            1.4675e-01, 6.7936e-01, -1.2429e-01, -6.3794e-01, -2.9083e-02,
#                                            -6.0464e-01, 1.0855e-01, 6.5904e-01, 5.2163e+00, 7.5811e-02,
#                                            8.2149e-03, -3.0893e-02, -4.0532e-02, -4.5461e-02, 3.8929e-03,
#                                            7.3546e-02, -5.1845e-02, -2.2415e-02, 7.4109e-03, -4.0126e-02,
#                                            7.2162e-02, 3.4596e-02, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                            0.0000e+00], dtype=torch.float32)
#             self.state_std = torch.tensor([0.1170, 0.0548, 0.0683, 0.0856, 0.1434, 0.3606, 0.2035, 0.4071, 0.1488,
#                                            0.3565, 0.1285, 0.4071, 0.1953, 1.2645, 1.0212, 1.1494, 1.6127, 1.8113,
#                                            1.3163, 4.3250, 3.2312, 5.4796, 2.4919, 4.3622, 2.3617, 5.3836, 3.0482,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#                                            0.0000, 0.0000, 0.0000], dtype=torch.float32)
#
#             # 4 runs
#             # self.state_avg = th.tensor([6.1537e-01, 8.9688e-01, 2.1685e-02, -5.6615e-02, -3.6099e-01,
#             #                                5.5272e-02, 6.4884e-01, -1.1314e-01, -5.7535e-01, -1.1797e-01,
#             #                                -5.4735e-01, 1.2350e-01, 6.3261e-01, 5.0387e+00, -3.1005e-01,
#             #                                5.8508e-03, -4.0760e-03, -3.9709e-03, -4.0554e-02, -4.4973e-03,
#             #                                5.5552e-02, -7.7341e-02, -3.3138e-02, -8.2667e-03, -2.2928e-02,
#             #                                6.2883e-02, 3.0411e-02, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#             #                                0.0000e+00], dtype=th.float32)
#             # self.state_std = th.tensor([0.1276, 0.0580, 0.0686, 0.0839, 0.1335, 0.3699, 0.2019, 0.4514, 0.1049,
#             #                                0.1996, 0.0715, 0.4507, 0.1640, 1.3036, 1.0192, 1.2708, 1.6660, 1.5512,
#             #                                1.2885, 4.3279, 3.5145, 6.1747, 2.1667, 2.8137, 1.4356, 6.1903, 2.8142,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,
#             #                                0.0000, 0.0000, 0.0000], dtype=th.float32)
#         elif env_name == 'HalfCheetah-v3':
#             self.env_num = 1
#             self.env_name = env_name
#             self.max_step = 1000
#             self.state_dim = 17
#             self.action_dim = 6
#             self.if_discrete = False
#             self.target_return = 5000
#
#             # 2 runs
#             self.state_avg = torch.tensor([-0.1786, 0.8515, 0.0683, 0.0049, 0.0143, -0.1074, -0.1226, -0.1223,
#                                            3.2042, -0.0244, 0.0103, 0.0679, -0.1574, 0.0661, -0.0098, 0.0513,
#                                            -0.0142], dtype=torch.float32)
#             self.state_std = torch.tensor([0.1224, 0.6781, 0.3616, 0.3545, 0.3379, 0.4800, 0.3575, 0.3372,
#                                            1.3460, 0.7967, 2.2092, 9.1078, 9.4349, 9.4631, 11.0645, 9.3995,
#                                            8.6867], dtype=torch.float32)
#         elif env_name == 'Walker2d-v3':
#             self.env_num = 1
#             self.env_name = env_name
#             self.max_step = 1000
#             self.state_dim = 17
#             self.action_dim = 6
#             self.if_discrete = False
#             self.target_return = 8000
#
#             # 6 runs
#             self.state_avg = torch.tensor([1.2954, 0.4176, -0.0995, -0.2242, 0.2234, -0.2319, -0.3035, -0.0614,
#                                            3.7896, -0.1081, 0.1643, -0.0470, -0.1533, -0.0410, -0.1140, -0.2981,
#                                            -0.6278], dtype=torch.float32)
#             self.state_std = torch.tensor([0.1095, 0.1832, 0.1664, 0.2951, 0.6291, 0.2582, 0.3270, 0.6931, 1.1162,
#                                            1.0560, 2.7070, 3.1108, 4.4344, 6.4363, 3.1945, 4.4594, 6.0115],
#                                           dtype=torch.float32)
#             # 11 runs
#             # self.state_avg = th.tensor([1.2026, 0.3181, -0.2361, -0.6064, -0.0210, -0.2863, -0.3759, -0.0214,
#             #                                4.7048, -0.0621, -0.0452, -0.1847, -0.6116, 0.0934, -0.0572, -0.5106,
#             #                                -0.5421], dtype=th.float32)
#             # self.state_std = th.tensor([0.0975, 0.2671, 0.2845, 0.6044, 0.6855, 0.3448, 0.4304, 0.7049, 1.5023,
#             #                                1.0364, 3.8605, 4.0202, 5.9124, 6.7366, 4.3993, 5.2269, 6.5471],
#             #                               dtype=th.float32)
#         else:
#             self.state_avg = torch.zeros(1, dtype=torch.float32)
#             self.state_std = torch.ones(1, dtype=torch.float32)
#             print(f"{self.__class__.__name__} WARNING: env_name not found {env_name}")
#
#         self.state_std = torch.clamp(self.state_std, 2 ** -4, 2 ** 4)  # todo
#         print(f'\n| {self.__class__.__name__}: We modified MuJoCo Env and do norm for state to make it better.')
#
#     def get_state_norm(self, state: Array) -> TEN:
#         state = torch.tensor(state, dtype=torch.float32)
#         return (state - self.state_avg) / self.state_std
#
#     def reset(self, **kwargs) -> Tuple[TEN, dict]:
#         state = self.env.reset(**kwargs)
#         return self.get_state_norm(state)
#
#     def step(self, action: Array) -> (TEN, float, bool, dict):
#         state, reward, done, info_dict = self.env.step(action)  # state, reward, done, info_dict
#         return self.get_state_norm(state), reward, done, info_dict

# PLAN TODO
# class HumanoidEnv(gym.Wrapper):  # [ElegantRL.2021.11.11]
#     def __init__(self, gym_env_id='Humanoid-v3', target_return=8000):
#         gym.logger.set_level(40)  # Block warning
#         super(HumanoidEnv, self).__init__(env=gym.make(gym_env_id))
#
#         # from elegantrl.envs.Gym import get_gym_env_info
#         # get_gym_env_info(env, if_print=True)  # use this function to print the env information
#         self.env_num = 1  # the env number of VectorEnv is greater than 1
#         self.env_name = gym_env_id  # the name of this env.
#         self.max_step = 1000  # the max step of each episode
#         self.state_dim = 376  # feature number of state
#         self.action_dim = 17  # feature number of action
#         self.if_discrete = False  # discrete action or continuous action
#         self.target_return = target_return  # episode return is between (-1600, 0)
#
#         # 5 runs
#         # self.state_avg = th.tensor([1.2027e+00, 9.0388e-01, -1.0409e-01, 4.4935e-02, -2.8785e-02,
#         #                                2.9601e-01, -3.1656e-01, 3.0909e-01, -4.3196e-02, -1.2750e-01,
#         #                                -2.6788e-01, -1.1086e+00, -1.1024e-01, 1.2908e-01, -5.8439e-01,
#         #                                -1.6043e+00, 8.1362e-02, -7.7958e-01, -4.3869e-01, -4.9594e-02,
#         #                                6.4827e-01, -3.0660e-01, 3.4619e+00, -5.2682e-02, -7.4712e-02,
#         #                                -5.4782e-02, 4.0784e-02, 1.3942e-01, 1.1000e-01, -1.3992e-02,
#         #                                9.3216e-02, -1.3473e-01, -7.6183e-02, -3.0072e-01, -1.3914e+00,
#         #                                -7.6460e-02, 1.6543e-02, -2.1907e-01, -3.8219e-01, -1.0018e-01,
#         #                                -1.5629e-01, -1.0627e-01, -3.7252e-03, 2.1453e-01, 2.7610e-02,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                1.5680e+00, 1.5862e+00, 1.9913e-01, 9.9125e-03, 5.5228e-03,
#         #                                -1.0950e-01, -1.2668e-01, 2.9367e-01, 3.5102e+00, 8.4719e+00,
#         #                                6.4141e-02, 6.6425e-02, 2.3180e-02, -2.3346e-03, 4.5395e-03,
#         #                                8.0720e-03, -3.0787e-02, -5.2109e-02, 3.2192e-01, 2.0724e+00,
#         #                                5.9864e-02, 5.0491e-02, 7.7832e-02, -3.2226e-03, 1.7504e-04,
#         #                                -1.9180e-03, -8.2688e-02, -1.9763e-01, 1.0849e-02, 5.9581e+00,
#         #                                2.5272e-01, 2.6957e-01, 1.1540e-01, 1.6143e-02, 2.7386e-02,
#         #                                -6.4959e-02, 2.4176e-01, -4.1101e-01, -8.2298e-01, 4.6070e+00,
#         #                                6.3743e-01, 7.0587e-01, 1.2301e-01, -4.3697e-04, -4.5899e-02,
#         #                                -6.8465e-02, 2.5412e-02, -1.7718e-01, -1.2062e+00, 2.6798e+00,
#         #                                6.8834e-01, 7.6378e-01, 1.2859e-01, -8.0863e-03, -1.0989e-01,
#         #                                -4.6906e-02, -1.4599e-01, -1.0927e-01, -1.0181e+00, 1.7989e+00,
#         #                                1.9099e-01, 2.0230e-01, 9.9341e-02, -1.5814e-02, 1.5009e-02,
#         #                                5.1159e-02, 1.6290e-01, 3.2563e-01, -6.0960e-01, 4.6070e+00,
#         #                                4.5602e-01, 4.9681e-01, 1.0787e-01, -5.9067e-04, -3.5140e-02,
#         #                                7.0788e-02, 2.5216e-02, 2.1480e-01, -9.1849e-01, 2.6798e+00,
#         #                                4.6612e-01, 5.2530e-01, 9.9732e-02, 1.3496e-02, -8.3317e-02,
#         #                                4.6769e-02, -1.8264e-01, 1.1677e-01, -7.7112e-01, 1.7989e+00,
#         #                                2.9806e-01, 2.7976e-01, 1.1250e-01, 3.8320e-03, 1.4312e-03,
#         #                                9.2314e-02, -2.9700e-02, -2.5973e-01, 5.9897e-01, 1.6228e+00,
#         #                                2.1239e-01, 1.6878e-01, 1.8192e-01, 6.9662e-03, -2.5374e-02,
#         #                                7.5638e-02, 3.0046e-02, -3.1797e-01, 2.8894e-01, 1.2199e+00,
#         #                                2.5424e-01, 2.0008e-01, 1.0215e-01, 1.6763e-03, -1.8978e-03,
#         #                                -8.9815e-02, -5.8642e-03, 3.2081e-01, 4.9344e-01, 1.6228e+00,
#         #                                1.8071e-01, 1.4553e-01, 1.4435e-01, -1.2074e-02, -1.3314e-02,
#         #                                -3.5878e-02, 5.3603e-02, 2.7511e-01, 2.0549e-01, 1.2199e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 4.2954e-02, 6.7016e-02, -2.8482e-02, 3.5978e+00,
#         #                                -4.8962e-02, -5.6775e-02, -4.4155e-02, -1.1466e-01, 1.6261e-01,
#         #                                3.5054e+00, -5.0701e-02, -4.9236e-02, -4.1256e-02, 1.1351e-01,
#         #                                1.3945e-01, 3.4389e+00, -4.3797e-02, -3.3252e-02, 2.8187e-02,
#         #                                -3.3888e-02, -3.5859e-01, 3.5962e+00, -3.8793e-02, -2.0773e-02,
#         #                                -2.4524e-02, 1.1582e+00, -4.5108e-02, 5.1413e+00, -8.7558e-02,
#         #                                -5.7185e-01, -2.4524e-02, 1.1582e+00, -4.5108e-02, 5.1413e+00,
#         #                                -8.7558e-02, -5.7185e-01, 9.9391e-02, -2.4059e-02, -1.7425e-01,
#         #                                3.4541e+00, -8.4718e-02, 1.8192e-02, 4.4070e-01, 3.9781e-01,
#         #                                3.5545e-01, 4.3428e+00, -1.8370e-01, -6.5439e-01, 4.4070e-01,
#         #                                3.9781e-01, 3.5545e-01, 4.3428e+00, -1.8370e-01, -6.5439e-01,
#         #                                1.5922e-01, 2.0918e-01, -9.8105e-02, 3.7604e+00, -2.9619e-02,
#         #                                -5.8485e-02, 1.0385e-01, 2.1228e-01, -1.7878e-01, 3.7999e+00,
#         #                                -7.4080e-02, -5.3348e-02, -2.6477e-01, 4.1909e-01, 2.9927e-02,
#         #                                3.6885e+00, -1.1708e-01, -6.7030e-02, -2.1599e-01, 3.9669e-01,
#         #                                6.0856e-03, 3.8305e+00, -8.3960e-02, -1.1403e-01, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                1.6677e+01, 4.9107e+01, -9.6274e+00, -2.9728e+01, -5.9374e+01,
#         #                                7.3201e+01, -5.8161e+01, -3.6315e+01, 2.7580e+01, 4.1244e+00,
#         #                                1.1711e+02, -8.4357e+00, -1.0379e+01, 1.0683e+01, 3.3124e+00,
#         #                                5.4840e+00, 8.2456e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00], dtype=th.float32)
#         # self.state_std = th.tensor([3.7685e-02, 4.5415e-02, 6.8201e-02, 9.5235e-02, 1.2801e-01, 2.2247e-01,
#         #                                2.2774e-01, 1.9151e-01, 1.0900e-01, 1.8950e-01, 3.8430e-01, 6.4591e-01,
#         #                                1.1708e-01, 1.7833e-01, 4.0411e-01, 6.1461e-01, 2.8869e-01, 3.0227e-01,
#         #                                4.4105e-01, 3.1090e-01, 3.5227e-01, 2.9399e-01, 8.6883e-01, 3.8865e-01,
#         #                                4.2435e-01, 2.4784e+00, 3.5310e+00, 4.3277e+00, 8.6461e+00, 6.9988e+00,
#         #                                7.2420e+00, 8.6105e+00, 9.3459e+00, 2.6776e+01, 4.3671e+01, 7.4211e+00,
#         #                                1.0446e+01, 1.4800e+01, 2.2152e+01, 5.7955e+00, 6.3750e+00, 7.0280e+00,
#         #                                6.4058e+00, 9.1694e+00, 7.0480e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 1.5156e-01, 1.4699e-01, 7.1690e-02, 3.6331e-02, 1.4871e-01,
#         #                                1.1873e-01, 3.9063e-01, 3.2367e-01, 1.9474e-01, 0.0000e+00, 1.1069e-02,
#         #                                1.1819e-02, 5.2432e-03, 3.1321e-03, 8.8166e-03, 6.7725e-03, 5.4790e-02,
#         #                                4.3172e-02, 3.4676e-02, 0.0000e+00, 1.0203e-02, 1.2745e-02, 1.4526e-02,
#         #                                9.4642e-03, 5.2404e-03, 5.6170e-03, 1.5328e-01, 1.1638e-01, 1.0253e-01,
#         #                                0.0000e+00, 4.8770e-02, 4.3080e-02, 4.4482e-02, 2.2124e-02, 4.9892e-02,
#         #                                2.2123e-02, 2.4277e-01, 1.0974e-01, 1.2796e-01, 0.0000e+00, 1.5967e-01,
#         #                                1.5963e-01, 6.8688e-02, 3.1619e-02, 1.2107e-01, 5.2330e-02, 2.8835e-01,
#         #                                1.1818e-01, 1.9899e-01, 0.0000e+00, 2.0831e-01, 2.2797e-01, 9.6549e-02,
#         #                                3.5202e-02, 1.2134e-01, 5.9960e-02, 2.1897e-01, 1.0345e-01, 2.1384e-01,
#         #                                0.0000e+00, 4.7938e-02, 4.4530e-02, 3.8997e-02, 2.2406e-02, 4.1815e-02,
#         #                                2.0735e-02, 2.1493e-01, 1.0405e-01, 1.4387e-01, 0.0000e+00, 1.5225e-01,
#         #                                1.6402e-01, 6.2498e-02, 3.1570e-02, 1.1685e-01, 4.3421e-02, 2.8339e-01,
#         #                                1.0626e-01, 2.1353e-01, 0.0000e+00, 1.9867e-01, 2.2000e-01, 8.5643e-02,
#         #                                3.0187e-02, 1.2717e-01, 5.0311e-02, 2.2468e-01, 9.0330e-02, 2.1959e-01,
#         #                                0.0000e+00, 4.6455e-02, 4.4841e-02, 2.4198e-02, 1.8876e-02, 3.3907e-02,
#         #                                2.6701e-02, 9.6149e-02, 7.2464e-02, 6.3727e-02, 0.0000e+00, 6.9340e-02,
#         #                                6.5581e-02, 5.0208e-02, 3.8457e-02, 3.7162e-02, 3.9005e-02, 1.2357e-01,
#         #                                9.5124e-02, 1.0308e-01, 0.0000e+00, 4.5508e-02, 4.2817e-02, 2.3776e-02,
#         #                                2.1004e-02, 3.2342e-02, 2.5299e-02, 1.0703e-01, 7.1359e-02, 6.8018e-02,
#         #                                0.0000e+00, 5.5628e-02, 5.4957e-02, 4.5547e-02, 3.1943e-02, 3.2783e-02,
#         #                                2.8549e-02, 1.1968e-01, 9.6011e-02, 9.6069e-02, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 2.7535e+00,
#         #                                3.3878e+00, 4.3144e+00, 1.3207e+00, 8.2827e-01, 4.9262e-01, 4.4686e+00,
#         #                                3.7037e+00, 6.4924e+00, 9.9583e-01, 6.8091e-01, 4.7597e-01, 4.3920e+00,
#         #                                4.7409e+00, 6.0906e+00, 9.3958e-01, 4.9473e-01, 4.9569e-01, 7.5115e+00,
#         #                                1.5371e+01, 1.1053e+01, 1.2450e+00, 7.6206e-01, 1.0601e+00, 9.2410e+00,
#         #                                2.3707e+01, 1.0356e+01, 6.6857e+00, 2.4551e+00, 2.8653e+00, 9.2410e+00,
#         #                                2.3707e+01, 1.0356e+01, 6.6857e+00, 2.4551e+00, 2.8653e+00, 5.3753e+00,
#         #                                8.6029e+00, 8.1809e+00, 1.1586e+00, 5.8827e-01, 8.2327e-01, 6.3651e+00,
#         #                                1.1362e+01, 8.7067e+00, 4.3533e+00, 1.4509e+00, 2.1305e+00, 6.3651e+00,
#         #                                1.1362e+01, 8.7067e+00, 4.3533e+00, 1.4509e+00, 2.1305e+00, 4.5383e+00,
#         #                                5.4198e+00, 5.3263e+00, 2.0749e+00, 1.5746e+00, 8.2220e-01, 5.7299e+00,
#         #                                6.2163e+00, 6.0368e+00, 2.1437e+00, 1.8280e+00, 1.2940e+00, 5.5326e+00,
#         #                                5.0856e+00, 5.3383e+00, 1.7817e+00, 1.5361e+00, 8.9927e-01, 6.1037e+00,
#         #                                6.5608e+00, 6.2712e+00, 1.9360e+00, 1.6504e+00, 1.1001e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 1.7051e+02,
#         #                                1.5721e+02, 1.7507e+02, 1.7297e+02, 1.3840e+02, 5.2837e+02, 2.8931e+02,
#         #                                1.6753e+02, 1.6898e+02, 5.0561e+02, 3.0826e+02, 2.2299e+01, 2.6949e+01,
#         #                                2.4568e+01, 2.5537e+01, 2.9878e+01, 2.6547e+01, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#         #                                0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00], dtype=th.float32)
#         # 8 runs
#         self.state_avg = torch.tensor([1.2108e+00, 9.0593e-01, -1.2575e-01, 4.8278e-02, -4.7363e-02,
#                                        3.0758e-01, -3.6351e-01, 3.3824e-01, -4.4513e-02, -9.5673e-02,
#                                        -2.6830e-01, -1.0654e+00, -1.1868e-01, 1.6859e-01, -6.7167e-01,
#                                        -1.7219e+00, 1.7098e-01, -7.5045e-01, -3.6428e-01, -4.5543e-02,
#                                        6.9729e-01, -4.1325e-01, 3.3065e+00, -4.8535e-02, -8.7482e-02,
#                                        -7.2437e-02, 8.0267e-02, 1.1422e-01, 6.3917e-02, -4.3369e-02,
#                                        1.0969e-01, -1.7911e-01, -2.4718e-02, -4.7037e-01, -1.8689e+00,
#                                        -3.3888e-02, 3.1659e-02, -1.8880e-01, -4.1088e-01, -4.3491e-02,
#                                        -1.4319e-01, -2.2842e-02, -2.9954e-02, 3.3196e-01, -2.8202e-02,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        1.5682e+00, 1.5911e+00, 2.1484e-01, 1.3796e-02, 2.4396e-02,
#                                        -1.2319e-01, -1.8833e-01, 3.3373e-01, 3.5928e+00, 8.5499e+00,
#                                        6.5384e-02, 6.8184e-02, 2.5217e-02, -3.2912e-03, 7.6363e-03,
#                                        9.5407e-03, -5.0383e-02, -6.1559e-02, 3.2461e-01, 2.0915e+00,
#                                        6.2163e-02, 5.2387e-02, 8.1777e-02, -4.5177e-03, 3.6478e-04,
#                                        -1.9426e-03, -1.1423e-01, -2.2379e-01, 1.0095e-02, 6.0130e+00,
#                                        2.5295e-01, 2.6585e-01, 1.1579e-01, 1.5304e-02, 2.3657e-02,
#                                        -6.4327e-02, 2.3218e-01, -4.2584e-01, -8.2816e-01, 4.6494e+00,
#                                        6.5633e-01, 7.2495e-01, 1.2384e-01, 2.1672e-03, -3.9951e-02,
#                                        -6.5070e-02, 4.8542e-02, -1.7317e-01, -1.2392e+00, 2.7045e+00,
#                                        7.2061e-01, 7.9773e-01, 1.2873e-01, -4.7357e-03, -9.5605e-02,
#                                        -4.3178e-02, -1.1211e-01, -1.0523e-01, -1.0520e+00, 1.8155e+00,
#                                        1.8269e-01, 1.9581e-01, 9.5197e-02, -1.4370e-02, 1.2924e-02,
#                                        4.5945e-02, 1.7367e-01, 3.0414e-01, -5.7666e-01, 4.6494e+00,
#                                        4.4517e-01, 4.8470e-01, 1.0228e-01, -1.4548e-03, -3.1125e-02,
#                                        6.3631e-02, 5.2045e-02, 2.0269e-01, -8.8813e-01, 2.7045e+00,
#                                        4.6257e-01, 5.1419e-01, 9.1832e-02, 1.2465e-02, -7.4154e-02,
#                                        4.2695e-02, -1.6522e-01, 1.1613e-01, -7.5953e-01, 1.8155e+00,
#                                        2.9357e-01, 2.7974e-01, 1.1385e-01, 6.3163e-03, 3.7935e-03,
#                                        8.7228e-02, -3.7212e-02, -2.4926e-01, 5.9919e-01, 1.6377e+00,
#                                        2.0201e-01, 1.6933e-01, 1.7335e-01, 8.5288e-03, -2.7483e-02,
#                                        7.0444e-02, 3.2598e-02, -2.9903e-01, 2.8950e-01, 1.2311e+00,
#                                        2.4482e-01, 1.9030e-01, 1.0209e-01, 6.4776e-04, -3.0012e-03,
#                                        -8.5235e-02, -3.3090e-03, 3.2367e-01, 4.7833e-01, 1.6377e+00,
#                                        1.8395e-01, 1.4705e-01, 1.4698e-01, -1.1777e-02, -1.3145e-02,
#                                        -3.0231e-02, 4.9042e-02, 2.7848e-01, 1.9004e-01, 1.2311e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 5.9201e-02, 9.9734e-02, -6.5489e-02, 3.5387e+00,
#                                        -4.5762e-02, -6.9222e-02, -6.2818e-03, -5.8119e-02, 8.4466e-02,
#                                        3.3673e+00, -4.9401e-02, -6.1099e-02, 1.7103e-02, 1.5226e-01,
#                                        9.0974e-02, 3.2815e+00, -4.2284e-02, -4.6787e-02, 1.7020e-01,
#                                        -6.6382e-03, -2.6923e-01, 3.5500e+00, -4.1203e-02, -3.6445e-02,
#                                        -2.4479e-01, 1.1339e+00, 1.1257e-01, 5.9749e+00, -4.5722e-02,
#                                        -6.0756e-01, -2.4479e-01, 1.1339e+00, 1.1257e-01, 5.9749e+00,
#                                        -4.5722e-02, -6.0756e-01, 9.5687e-02, 6.2377e-03, -3.1253e-01,
#                                        3.3551e+00, -7.5612e-02, 3.7902e-03, 4.9319e-01, 4.9548e-01,
#                                        7.1103e-02, 4.4660e+00, -1.7679e-01, -5.6680e-01, 4.9319e-01,
#                                        4.9548e-01, 7.1103e-02, 4.4660e+00, -1.7679e-01, -5.6680e-01,
#                                        2.4826e-01, 2.7281e-01, -5.3309e-02, 3.8251e+00, -6.9774e-03,
#                                        -7.2389e-02, 1.6979e-01, 2.6176e-01, -7.4322e-02, 3.8449e+00,
#                                        -5.5816e-02, -8.0149e-02, -2.8148e-01, 4.7921e-01, 7.2474e-03,
#                                        3.7309e+00, -1.1763e-01, -7.3255e-02, -3.3529e-01, 5.1496e-01,
#                                        -2.1279e-02, 3.9610e+00, -9.3358e-02, -1.1908e-01, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        -3.1586e+00, 5.5951e+01, -5.1688e+00, -2.0856e+01, -9.7607e+00,
#                                        1.0722e+02, 3.5213e+01, 1.2223e+01, 3.3327e+01, -6.1532e+01,
#                                        1.0860e+02, 2.3747e+00, -9.9348e+00, 1.9073e+01, -2.5358e-01,
#                                        1.0303e+01, 3.9810e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00], dtype=torch.float32)
#         self.state_std = torch.tensor([3.1392e-02, 3.7876e-02, 5.7367e-02, 8.0870e-02, 1.1060e-01, 1.9527e-01,
#                                        1.9386e-01, 1.6299e-01, 9.1475e-02, 1.6569e-01, 3.3626e-01, 5.7772e-01,
#                                        9.8797e-02, 1.5675e-01, 3.4861e-01, 5.3704e-01, 2.4981e-01, 2.8493e-01,
#                                        3.7375e-01, 2.8416e-01, 3.1271e-01, 2.7643e-01, 8.0354e-01, 3.5500e-01,
#                                        3.8352e-01, 2.5504e+00, 3.5465e+00, 4.5748e+00, 9.9119e+00, 7.4044e+00,
#                                        7.7644e+00, 9.2271e+00, 1.2024e+01, 3.4966e+01, 7.6774e+01, 8.9679e+00,
#                                        1.2524e+01, 1.5256e+01, 2.4119e+01, 6.6164e+00, 7.7344e+00, 7.6366e+00,
#                                        7.4501e+00, 9.9335e+00, 1.0271e+01, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 1.3350e-01, 1.2936e-01, 6.1098e-02, 3.0401e-02, 1.2934e-01,
#                                        1.0183e-01, 3.6737e-01, 2.9502e-01, 1.7495e-01, 0.0000e+00, 9.3017e-03,
#                                        9.9151e-03, 4.3507e-03, 2.5781e-03, 7.3343e-03, 5.6374e-03, 4.5623e-02,
#                                        3.5967e-02, 2.9087e-02, 0.0000e+00, 8.4761e-03, 1.0561e-02, 1.2082e-02,
#                                        7.8437e-03, 4.3507e-03, 4.7044e-03, 1.3116e-01, 9.9861e-02, 8.7706e-02,
#                                        0.0000e+00, 4.0845e-02, 3.5960e-02, 3.7137e-02, 1.8467e-02, 4.1754e-02,
#                                        1.8461e-02, 2.0857e-01, 9.3311e-02, 1.1018e-01, 0.0000e+00, 1.3535e-01,
#                                        1.3499e-01, 5.7386e-02, 2.6211e-02, 1.0184e-01, 4.3831e-02, 2.4708e-01,
#                                        1.0116e-01, 1.7149e-01, 0.0000e+00, 1.7665e-01, 1.9352e-01, 8.0759e-02,
#                                        2.9150e-02, 1.0249e-01, 5.0426e-02, 1.8785e-01, 8.8313e-02, 1.8192e-01,
#                                        0.0000e+00, 4.0083e-02, 3.7293e-02, 3.2804e-02, 1.8966e-02, 3.5081e-02,
#                                        1.7295e-02, 1.8324e-01, 9.0613e-02, 1.2549e-01, 0.0000e+00, 1.2928e-01,
#                                        1.3884e-01, 5.2734e-02, 2.6758e-02, 9.7935e-02, 3.6490e-02, 2.4088e-01,
#                                        9.2416e-02, 1.8560e-01, 0.0000e+00, 1.6895e-01, 1.8693e-01, 7.2220e-02,
#                                        2.5554e-02, 1.0662e-01, 4.2373e-02, 1.9043e-01, 7.6969e-02, 1.8709e-01,
#                                        0.0000e+00, 3.8692e-02, 3.7292e-02, 2.0231e-02, 1.5743e-02, 2.8439e-02,
#                                        2.2167e-02, 8.2798e-02, 6.1350e-02, 5.3325e-02, 0.0000e+00, 5.7712e-02,
#                                        5.4608e-02, 4.1780e-02, 3.1920e-02, 3.0993e-02, 3.2481e-02, 1.0455e-01,
#                                        7.9928e-02, 8.6195e-02, 0.0000e+00, 3.8208e-02, 3.5703e-02, 1.9769e-02,
#                                        1.7572e-02, 2.7208e-02, 2.1206e-02, 9.1823e-02, 6.0944e-02, 5.7677e-02,
#                                        0.0000e+00, 4.6909e-02, 4.6119e-02, 3.8016e-02, 2.6753e-02, 2.7516e-02,
#                                        2.3943e-02, 1.0474e-01, 8.3302e-02, 8.2673e-02, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 2.8122e+00,
#                                        3.5878e+00, 4.3692e+00, 1.2417e+00, 7.6688e-01, 4.4274e-01, 4.5708e+00,
#                                        4.0605e+00, 7.5402e+00, 9.2691e-01, 6.3951e-01, 4.2625e-01, 4.5672e+00,
#                                        4.9948e+00, 6.9801e+00, 8.6809e-01, 4.5658e-01, 4.4578e-01, 8.1632e+00,
#                                        1.9371e+01, 1.5762e+01, 1.2160e+00, 7.1688e-01, 9.8074e-01, 1.0888e+01,
#                                        3.5036e+01, 1.4247e+01, 9.0313e+00, 2.8052e+00, 3.3265e+00, 1.0888e+01,
#                                        3.5036e+01, 1.4247e+01, 9.0313e+00, 2.8052e+00, 3.3265e+00, 6.0875e+00,
#                                        9.3019e+00, 9.6741e+00, 1.1009e+00, 5.3437e-01, 7.4614e-01, 7.1663e+00,
#                                        1.2823e+01, 1.0369e+01, 4.1288e+00, 1.3454e+00, 2.0126e+00, 7.1663e+00,
#                                        1.2823e+01, 1.0369e+01, 4.1288e+00, 1.3454e+00, 2.0126e+00, 5.1024e+00,
#                                        6.0538e+00, 5.7377e+00, 2.0800e+00, 1.5886e+00, 7.5714e-01, 6.4385e+00,
#                                        7.0912e+00, 6.6091e+00, 2.1412e+00, 1.8227e+00, 1.1804e+00, 6.2504e+00,
#                                        5.4816e+00, 5.8103e+00, 1.7573e+00, 1.5686e+00, 8.4100e-01, 7.1933e+00,
#                                        8.0470e+00, 7.3113e+00, 1.9905e+00, 1.7208e+00, 1.1594e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 2.5147e+02,
#                                        2.3960e+02, 2.5526e+02, 2.2783e+02, 2.2161e+02, 7.6230e+02, 4.4391e+02,
#                                        2.2895e+02, 2.4944e+02, 7.0961e+02, 4.6304e+02, 3.2988e+01, 3.9116e+01,
#                                        3.1438e+01, 3.6047e+01, 4.0998e+01, 3.7787e+01, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00,
#                                        0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00], dtype=torch.float32)
#         # self.state_avg = th.zeros(1, dtype=th.float32)
#         # self.state_std = th.ones(1, dtype=th.float32)
#
#         self.state_std = torch.clamp(self.state_std, 2 ** -4, 2 ** 4)
#         print(f'\n| {self.__class__.__name__}: We modified MuJoCo Env and do norm for state to make it better.'
#               f'\n| We scale the action space from (-0.4, +0.4), to (-1, +1).')
#
#     def get_state_norm(self, state: Array) -> TEN:
#         state = torch.tensor(state, dtype=torch.float32)
#         return (state - self.state_avg) / self.state_std
#
#     def reset(self) -> TEN:
#         state = self.env.reset()
#         return self.get_state_norm(state)
#
#     def step(self, action: Array) -> (TEN, float, bool, dict):
#         # MuJoCo Humanoid Env set its action space as (-0.4, +0.4). It is bad.
#         # I suggest to set action space as (-1, +1) when you design your own env.
#         # action_space.high = 0.4
#         # action_space.low = -0.4
#         state, reward, done, info_dict = self.env.step(action * 2.5)  # state, reward, done, info_dict
#         return self.get_state_norm(state), reward, done, info_dict
